/*************************************************************************
 * Copyright (C) [2018] by Cambricon, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *************************************************************************/

#include <stdio.h>
#include <string.h>
#include <cndev.h>
#ifndef WIN32
#include <sys/sysinfo.h>
#endif
int printName(cndevTopologyNode_t* treeNode, void* num) {
  (void) num;
  int ret = 1;
  printf("callback func\n");
  if (treeNode->domain == 0 && treeNode->bus == 0 && treeNode->device == 0) {
    printf("traverse tree test:\nthe bdf of node is %04x:%02x:%02x.%x\n", treeNode->domain,
           treeNode->bus, treeNode->device, treeNode->function);
    ret = 0;
  }
  return ret;
}
int main() {
  // get lib version, don't need to init cndev
  cndevLibVersionInfo_t libVersion;
  libVersion.version = CNDEV_VERSION_5;
  cndevCheckErrors(cndevGetLibVersion(&libVersion));
  printf("cndev lib version:%d.%d.%d\n",
         libVersion.libMajorVersion,
         libVersion.libMinorVersion,
         libVersion.libBuildVersion);

  // init libcndev
  cndevRet_t ret = cndevInit(0);
  // you can compare cndevRet manually, or just use cndevCheckErrors
  if (CNDEV_SUCCESS != ret) {
    printf("cndev init failed: %s.\n", cndevGetErrorString(ret));
    // should exit now
    // exit(0);
  }
  // get card count
  cndevCardInfo_t cardInfo;
  cardInfo.version = CNDEV_VERSION_5;
  ret = cndevGetDeviceCount(&cardInfo);
  if (ret == CNDEV_ERROR_LOW_DRIVER_VERSION) {
    // get api's corresponding lowest support driver version
    cndevVersionInfo_t lowestDriverVersion;
    lowestDriverVersion.version = CNDEV_VERSION_5;
    // notice: when return value isn't CNDEV_SUCCESS, program will exit with 1
    cndevCheckErrors(cndevGetLowestSupportDriverVersion(&lowestDriverVersion));
    printf("api version %d, driver version should be >= %u.%u.%u\n",
           CNDEV_VERSION_5,
           lowestDriverVersion.driverMajorVersion,
           lowestDriverVersion.driverMinorVersion,
           lowestDriverVersion.driverBuildVersion);
  }
  cndevCheckErrors(ret);
  printf("card num = %d\n", cardInfo.number);

  for (unsigned int card = 0; card < cardInfo.number; card++) {
    // get device hanlde
    cndevDevice_t devHandle;
    ret = cndevGetDeviceHandleByIndex(card, &devHandle);
    cndevCheckErrors(ret);
    // ret = cndevGetDeviceHandleByPciBusId("0000:00:00.0", &devHandle);
    // cndevCheckErrors(ret);
    // ret = cndevGetDeviceHandleBySerial("0000000000000000", &devHandle);
    // cndevCheckErrors(ret);
    // ret = cndevGetDeviceHandleByUUID("00000000-0000-0000-0000-000000000000", &devHandle);
    // cndevCheckErrors(ret);

    // get card[x]'s name info
    cndevCardName_t nameInfo;
    nameInfo.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetCardName(&nameInfo, devHandle));
    printf("Card %u: %s\n", card, cndevGetCardNameString(nameInfo.id));
    // get cardname by devId
    printf("Card Name:%s\n", cndevGetCardNameStringByDevId(devHandle));

    // get card[x]'s memory info
    cndevMemoryInfo_t memInfo;
    memInfo.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetMemoryUsage(&memInfo, devHandle));
    printf("Phy mem total:%ldMiB, Phy mem used:%ldMiB, Vir mem total:%ldMiB, Vir mem used:%ldMiB\n"
           "Global mem:%ldMiB\n",
           memInfo.physicalMemoryTotal,
           memInfo.physicalMemoryUsed,
           memInfo.virtualMemoryTotal,
           memInfo.virtualMemoryUsed,
           memInfo.globalMemory);
    for (int i = 0; i < memInfo.channelNumber; ++i) {
      printf("Channel %d mem used:%ldMib\n", i, memInfo.channelMemoryUsed[i]);
    }

    // get card[x]'s mcu and driver's version
    cndevVersionInfo_t verInfo;
    verInfo.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetVersionInfo(&verInfo, devHandle));
    printf("MCU Version:%x.%x.%x\n"
           "Driver Version:%u.%u.%u\n",
           verInfo.mcuMajorVersion,
           verInfo.mcuMinorVersion,
           verInfo.mcuBuildVersion,
           verInfo.driverMajorVersion,
           verInfo.driverMinorVersion,
           verInfo.driverBuildVersion);

    // get health state
    cndevCardHealthState_t healthstate;
    healthstate.version = CNDEV_VERSION_5;
    ret = cndevGetCardHealthState(&healthstate, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetCardHealthState is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("HealthState:%d\n", healthstate.health);
    }

    // get card[x]'s MLU utilization
    cndevUtilizationInfo_t utilInfo;
    utilInfo.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetDeviceUtilizationInfo(&utilInfo, devHandle));
    printf("Util:%d%%\n", utilInfo.averageCoreUtilization);

    // get card[x]'s core count
    cndevCardCoreCount_t cardCoreCount;
    cardCoreCount.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetCoreCount(&cardCoreCount, devHandle));
    for (int i = 0; i < cardCoreCount.count; i++) {
      printf("Util CORE%d:%d%%\n", i, utilInfo.coreUtilization[i]);
    }

    // get card[x]'s frequency info
    cndevFrequencyInfo_t freqInfo;
    freqInfo.version = CNDEV_VERSION_5;
    ret = cndevGetFrequencyInfo(&freqInfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetFrequencyInfo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Core Freq:%d MHz\n", freqInfo.boardFreq);
      printf("Memory Freq:%d MHz\n", freqInfo.ddrFreq);
      printf("Over-temperature dynamic frequency: %d\n", freqInfo.overtempDfsFlag);
      printf("Fast dynamic frequency: %d\n", freqInfo.fastDfsFlag);
      printf("Defauluts: %d\n", freqInfo.boardDefaultFreq);
      printf("Adjustment range: %d - %d\n", freqInfo.boardFreqArange[0], freqInfo.boardFreqArange[1]);
      printf("MLU Cluster Freq Count:%d\n", freqInfo.mluClusterFreqCount);
      for (int i = 0; i < freqInfo.mluClusterFreqCount; ++i)
        printf("  MLU Cluster %d Freq:%d MHz\n", i, freqInfo.mluClusterFreq[i]);
    }

    // get card[x]'s ddr ecc info
    cndevECCInfo_t eccInfo;
    eccInfo.version = CNDEV_VERSION_5;
    ret = cndevGetECCInfo(&eccInfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetECCInfo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("ECC:one bit:%lu, mul one:%lu, mul:%lu, mul mul:%lu, correct error:%lu, un correct error:%lu, total:%lu\n",
           eccInfo.oneBitError, eccInfo.multipleOneError, eccInfo.multipleError,
           eccInfo.multipleMultipleError, eccInfo.correctedError, eccInfo.uncorrectedError,
           eccInfo.totalError);
      printf("addressforbidden error:%lu\n", eccInfo.addressForbiddenError);
    }

    // get card[x]'s power usage info
    cndevPowerInfo_t powerInfo;
    powerInfo.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetPowerInfo(&powerInfo, devHandle));
    printf("power:usage:%d.%d W, cap:%d W\n", powerInfo.usage, powerInfo.usageDecimal, powerInfo.cap);
    printf("machine power:%d W\n", powerInfo.machine);
    printf("thermal design power:%d W\n", powerInfo.thermalDesignPower);
    printf("Instantaneous Power Usage:%d.%d W\n", powerInfo.instantaneousPowerUsage, powerInfo.instantaneousPowerUsageDecimal);

    // get card[x]'s device power usage info
    cndevPowerInfo_t device_power;
    device_power.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetDevicePower(&device_power, devHandle));
    printf("power:usage:%d.%d W, cap:%d W\n", device_power.usage, device_power.usageDecimal, device_power.cap);
    printf("thermal design power:%d W\n", device_power.thermalDesignPower);
    printf("Instantaneous Power Usage:%d.%d W\n", device_power.instantaneousPowerUsage, device_power.instantaneousPowerUsageDecimal);

    // get card[x]'s power management limit
    cndevPowerManagementLimit_t limit;
    limit.version = CNDEV_VERSION_5;
    ret = cndevGetPowerManagementLimit(&limit, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetPowerManagementLimit is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Min Power Limit:%d.%02d W\n", limit.minPowerLimit, limit.minPowerLimitDecimal);
      printf("Max Power Limit:%d.%02d W\n", limit.maxPowerLimit, limit.maxPowerLimitDecimal);
    }

    // get card[x]'s fan speed info
    cndevFanSpeedInfo_t fanInfo;
    fanInfo.version = CNDEV_VERSION_5;
    ret = cndevGetFanSpeedInfo(&fanInfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetFanSpeedInfo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("fan:%d%%\n", fanInfo.fanSpeed);
      if (nameInfo.id == MLU290) {
        for (int i = 0; i < fanInfo.chassisFanCount; ++i) {
          printf("chassis fan %d:%d\n", i, fanInfo.chassisFan[i]);
        }
      }
    }

    // get card[x]'s cluster count
    cndevCardClusterCount_t cardClusterCount;
    cardClusterCount.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetClusterCount(&cardClusterCount, devHandle));

    // get card[x]'s MLU temperature info
    cndevTemperatureInfo_t tempInfo;
    tempInfo.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetTemperatureInfo(&tempInfo, devHandle));
    printf("temperatures:board:%dC\nClusters:", tempInfo.board);
    for (int i = 0; i < cardClusterCount.count; i++) {
      printf("%dC ", tempInfo.cluster[i]);
    }
    printf("\n");
    printf("Chip temperature:%d C\n", tempInfo.chip);
    printf("Air inlet temperature:%d C\n", tempInfo.airInlet);
    printf("Air outlet temperature:%d C\n", tempInfo.airOutlet);
    printf("Memory temperature:%d C\n", tempInfo.memory);
    printf("Video Input temperature:%d C\n", tempInfo.videoInput);
    printf("CPU temperature:%d C\n", tempInfo.cpu);
    printf("ISP temperature:%d C\n", tempInfo.isp);

    // get card[x]'s device temperature info
    cndevTemperatureInfo_t device_temp;
    device_temp.version = CNDEV_VERSION_5;
    cndevCheckErrors(cndevGetDeviceTemperature(&device_temp, devHandle));
    printf("temperatures:board:%dC\nClusters:", device_temp.board);
    for (int i = 0; i < cardClusterCount.count; i++) {
      printf("%dC ", device_temp.cluster[i]);
    }
    printf("\n");
    printf("Chip temperature:%d C\n", device_temp.chip);
    printf("Memory temperature:%d C\n", device_temp.memory);
    printf("Video Input temperature:%d C\n", device_temp.videoInput);
    printf("CPU temperature:%d C\n", device_temp.cpu);
    printf("ISP temperature:%d C\n", device_temp.isp);

    // get card[x]'s memorydie count
    cndevCardMemoryDieCount_t memorydiecount;
    memorydiecount.version = CNDEV_VERSION_5;
    ret = cndevGetMemoryDieCount(&memorydiecount, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetMemoryDieCount is not supported.\n");
    } else {
      printf("memorydie count:%d\n", memorydiecount.count);
      for (int i = 0; i < memorydiecount.count; i++) {
        printf("memoryDie %d temperature: %d C\n", i, tempInfo.memoryDie[i]);
      }
    }

    //get card[x]'s SN
    cndevCardSN_t cardSN;
    cardSN.version = CNDEV_VERSION_5;
    ret = cndevGetCardSN(&cardSN, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetCardSN is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      if (cardSN.sn==0)
        printf("The SN of the card:0\n");
      else
        printf("The SN of the card:%012lx\n", cardSN.sn);
      printf("The motherboard SN of the card:%012lx\n", cardSN.motherBoardSn);
    }

    // get device Id info
    cndevPCIeInfo_t deviceId;
    deviceId.version = CNDEV_VERSION_5;
    ret = cndevGetPCIeInfo(&deviceId, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetPCIeInfo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("subsystemId:0x%x\ndeviceId:0x%x\nvendor:0x%x\nsubsystemVendor:0x%x\nbusNum:%02x\ndomainId:%04x\nfunction:%x\ndevice:%02x\n",
           deviceId.subsystemId, deviceId.deviceId, deviceId.vendor, deviceId.subsystemVendor,deviceId.bus,
           deviceId.domain, deviceId.function, deviceId.device);
#ifndef WIN32
    printf("physical slot:%s\n", deviceId.physicalSlot);
#endif
    }

    //get PCIe throughput
    cndevPCIethroughput_t pciethroughput;
    pciethroughput.version = CNDEV_VERSION_5;
    ret = cndevGetPCIethroughput(&pciethroughput, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetPCIethroughput is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("pcieRead:%ld B\n", pciethroughput.pcieRead);
      printf("pcieWrite:%ld B\n", pciethroughput.pcieWrite);
    }

    // get true link speed
    cndevLinkSpeed_t linkSpeed;
    linkSpeed.version = CNDEV_VERSION_5;
    ret = cndevGetLowestLinkSpeed(&linkSpeed, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetLowestLinkSpeed is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      switch (linkSpeed.linkSpeed) {
        case 1:
          printf("pcie switch link speed:2.5 GT/s\n");
          break;
        case 2:
          printf("pcie switch link speed:5 GT/s\n");
          break;
        case 3:
          printf("pcie switch link speed:8 GT/s\n");
          break;
        case 4:
          printf("pcie switch link speed:16 GT/s\n");
          break;
        default:
          printf("pcie switch link speed: UNKNOWN\n");
          break;
      }
    }

    cndevCodecTurbo_t codecturbo;
    codecturbo.version = CNDEV_VERSION_5;
    ret = cndevGetCodecTurbo(&codecturbo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetCodecTurbo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      switch (codecturbo.codecTurbo) {
      case 1:
        printf("Codec Turbo : ENABLE\n");
        break;
      case 0:
        printf("Codec Turbo : DISABLE\n");
        break;
      default:
        printf("Codec Turbo : UNKNOWN\n");
        break;
      }
    }

    // get fast alloc memory info
    cndevFastAlloc_t fastalloc;
    fastalloc.version = CNDEV_VERSION_5;
    ret = cndevGetFastAlloc(&fastalloc, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetFastAlloc is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Fast Alloc Memtotal: %d KB\n", fastalloc.fastMemoryTotal);
      printf("Fast Alloc Memfree: %d KB\n", fastalloc.fastMemoryFree);
    }

    cndevVideoCodecUtilization_t vpuutil;
    vpuutil.version = CNDEV_VERSION_5;
    ret = cndevGetVideoCodecUtilization(&vpuutil, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetVideoCodecUtilization is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("VPU Count: %d \n", vpuutil.vpuCount);
      for (int i = 0; i < vpuutil.vpuCount; ++i) {
        printf("VPU Utilization: %d \n", vpuutil.vpuCodecUtilization[i]);
      }
      printf("VPU Decoder Count: %d \n", vpuutil.decoderCount);
      for (int i = 0; i< vpuutil.decoderCount; ++i) {
        printf("VPU Decoder Utilization: %d \n", vpuutil.decoderUtilization[i]);
      }
      printf("VPU Encoder Count: %d \n", vpuutil.encoderCount);
      for (int i = 0; i< vpuutil.encoderCount; ++i) {
        printf("VPU Encoder Utilization: %d \n", vpuutil.encoderUtilization[i]);
      }
    }

    cndevImageCodecUtilization_t jpuutil;
    jpuutil.version = CNDEV_VERSION_5;
    ret = cndevGetImageCodecUtilization(&jpuutil, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetImageCodecUtilization is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("JPU Count: %d \n", jpuutil.jpuCount);
      for (int i = 0; i < jpuutil.jpuCount; ++i) {
        printf("JPU Utilization: %d \n", jpuutil.jpuCodecUtilization[i]);
      }
    }

    cndevScalerUtilization_t scalerutil;
    scalerutil.version = CNDEV_VERSION_5;
    ret = cndevGetScalerUtilization(&scalerutil, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetScalerUtilization is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Scaler Count: %d \n", scalerutil.scalerCount);
      for (int i = 0; i < scalerutil.scalerCount; ++i) {
        printf("Scaler Utilization: %d \n", scalerutil.scalerUtilization[i]);
      }
    }

    // get card[x]'s qdd status
    cndevQsfpddStatus_t qddstatus;
    qddstatus.version = CNDEV_VERSION_5;
    ret = cndevGetQsfpddStatus(&qddstatus, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetQsfpddStatus is not supported.\n");
    } else {
      for (int i = 0; i < 8; i++) {
        printf("QDD %d:", i);
        if (qddstatus.qdd[i] == 1) {
          printf("Valid\n");
        } else {
          printf("Invalid\n");
        }
      }
    }

    // get card[x]'s chassis status
    cndevChassisInfo_t chassisinfo;
    chassisinfo.version = CNDEV_VERSION_5;
    ret = cndevGetChassisInfo(&chassisinfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetChassisInfo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Chassis part number:%s\n", chassisinfo.chassisPartNum);
      printf("Chassis vendor name:%s\n", chassisinfo.chassisVendorName);
      printf("Chassis product date:%s\n", chassisinfo.chassisProductDate);
      printf("Chassis sn:%lu\n", chassisinfo.chassisSn);
      for (int i = 0; i < chassisinfo.ibBoardNum; ++i) {
        printf("IB SN:%s\n", chassisinfo.ibInfo[i].ibSn);
        printf("IB FW:%s\n", chassisinfo.ibInfo[i].ibFw);
        printf("IB MFC:%s\n", chassisinfo.ibInfo[i].ibMfc);
        printf("IB Model:%s\n", chassisinfo.ibInfo[i].ibModel);
      }
      for (int i = 0; i < chassisinfo.nvmeSsdNum; ++i) {
        printf("NVME SN:%s\n", chassisinfo.nvmeInfo[i].nvmeSn);
        printf("NVME FW:%s\n", chassisinfo.nvmeInfo[i].nvmeFw);
        printf("NVME MFC:%s\n", chassisinfo.nvmeInfo[i].nvmeMfc);
        printf("NVME Model:%s\n", chassisinfo.nvmeInfo[i].nvmeModel);
      }
      for (int i = 0; i < chassisinfo.psuNum; ++i) {
        printf("PSU SN:%s\n", chassisinfo.psuInfo[i].psuSn);
        printf("PSU FW:%s\n", chassisinfo.psuInfo[i].psuFw);
        printf("PSU MFC:%s\n", chassisinfo.psuInfo[i].psuMfc);
        printf("PSU Model:%s\n", chassisinfo.psuInfo[i].psuModel);
      }
    }

    // get card[x]'s PCIe firmware version
    cndevPCIeFirmwareVersion_t pcie_fw_verison;
    pcie_fw_verison.version = CNDEV_VERSION_5;
    ret = cndevGetPCIeFirmwareVersion(&pcie_fw_verison, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetPCIeFirmwareVersion is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("PCIe firmware verison is V%04x.%04x.%03x\n",
              pcie_fw_verison.pcieReversion, pcie_fw_verison.pcieBuildID, pcie_fw_verison.pcieEngineeringId);
    }

    // get card[x]'s device CPU utilization
    cndevDeviceCPUUtilization_t cpu_util;
    cpu_util.version = CNDEV_VERSION_5;
    ret = cndevGetDeviceCPUUtilization(&cpu_util, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetDeviceCPUUtilization is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Device CPU Chip Utilization: %d%%\n", cpu_util.chipUtilization);
      for (int i = 0; i < cpu_util.coreNumber; ++i) {
        printf("Device CPU %d Utilization: %d%%\n", i, cpu_util.coreUtilization[i]);
      }
    }

    // set card[x]'s device CPU sampling interval
    // cndevDeviceCPUSamplingInterval_t set_time;
    // set_time.version = CNDEV_VERSION_5;
    // set_time.samplingInterval = 100;  // range from 10 to 10000 ms, default is 100 ms
    // ret = cndevSetDeviceCPUSamplingInterval(&set_time, devHandle);
    // if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
    //   printf("cndevGetDeviceCPUSamplingInterval is not supported.\n");
    // } else {
    //   cndevCheckErrors(ret);
    //   printf("Set Device CPU sampling frequency: %d ms\n", set_time.samplingInterval);
    // }

    // get card[x]'s device CPU sampling interval
    cndevDeviceCPUSamplingInterval_t interval;
    interval.version = CNDEV_VERSION_5;
    ret = cndevGetDeviceCPUSamplingInterval(&interval, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetDeviceCPUSamplingInterval is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Device CPU sampling frequency: %d ms\n", interval.samplingInterval);
    }

    // get card[x]'s tiny core utilization
    cndevTinyCoreUtilization_t tinycoreutil;
    tinycoreutil.version = CNDEV_VERSION_5;
    ret = cndevGetTinyCoreUtilization(&tinycoreutil, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetTinyCoreUtilization is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("card %d tinycore count: %d\n", card, tinycoreutil.tinyCoreCount);
      for (int i = 0; i < tinycoreutil.tinyCoreCount; ++i) {
        printf("tinycore %d: %d\n", i, tinycoreutil.tinyCoreUtilization[i]);
      }
    }

    // get card[x]'s arm os memory info
    cndevArmOsMemoryInfo_t armos_mem;
    armos_mem.version = CNDEV_VERSION_5;
    ret = cndevGetArmOsMemoryUsage(&armos_mem, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetArmOsMemoryUsage is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("card %d device system memory total: %ld KB\n", card, armos_mem.deviceSystemMemoryTotal);
      printf("card %d device system memory used: %ld KB\n", card, armos_mem.deviceSystemMemoryUsed);
    }

    // get card[x]'x MLU frequency status
    cndevMLUFrequencyStatus_t freq_status;
    freq_status.version = CNDEV_VERSION_5;
    ret = cndevGetMLUFrequencyStatus(&freq_status, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetMLUFrequencyStatus is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("card %d MLU frequency status: %s\n", card, freq_status.mluFrequencyLockStatus ? "Locked" : "Unlocked");
    }

    // get card[x]'s process info
    // in most cases, the number of processes running on a card will not exceed 10
    // but if cndevProcessInfo's space is not enough, CNDEV_ERROR_INSUFFICIENT_SPACE will be returned
    unsigned tCount = 10;
    cndevProcessInfo_t *procInfo = NULL;
    procInfo = (cndevProcessInfo_t *) malloc(tCount * sizeof(cndevProcessInfo_t));
    procInfo->version = CNDEV_VERSION_5;
    ret = cndevGetProcessInfo(&tCount, procInfo, devHandle);
    // if ret is CNDEV_ERROR_INSUFFICIENT_SPACE, should get again
    while (ret == CNDEV_ERROR_INSUFFICIENT_SPACE) {
      procInfo = (cndevProcessInfo_t *) realloc(procInfo, tCount * sizeof(cndevProcessInfo_t));
      ret = cndevGetProcessInfo(&tCount, procInfo, devHandle);
    }
    cndevCheckErrors(ret);
    printf("process: count: %d\n", tCount);

    for (unsigned int i = 0; i < tCount; i++) {
      printf("process %d:pid:%u, PhyMem:%luKiB, VirMem:%luKiB\n",
             i,
             (procInfo + i)->pid,
             (procInfo + i)->physicalMemoryUsed,
             (procInfo + i)->virtualMemoryUsed);
    }
    free(procInfo);

    // get card[x]'s process utilization
    // in most cases, the number of processes running on a card will not exceed 10
    // but if cndevProcessUtilization_t's space is not enough, CNDEV_ERROR_INSUFFICIENT_SPACE will be returned
    unsigned pCount = 10;
    cndevProcessUtilization_t *procUtil = NULL;
    procUtil = (cndevProcessUtilization_t *) malloc(pCount * sizeof(cndevProcessUtilization_t));
    procUtil->version = CNDEV_VERSION_5;
    ret = cndevGetProcessUtilization(&pCount, procUtil, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetProcessUtilization is not supported.\n");
    } else {
      // if ret is CNDEV_ERROR_INSUFFICIENT_SPACE, should get again
      while (ret == CNDEV_ERROR_INSUFFICIENT_SPACE) {
        procUtil = (cndevProcessUtilization_t *) realloc(procUtil, pCount * sizeof(cndevProcessUtilization_t));
        ret = cndevGetProcessUtilization(&pCount, procUtil, devHandle);
      }
      cndevCheckErrors(ret);
      printf("process: count: %d\n", pCount);
      for (unsigned int i = 0; i < pCount; i++) {
        printf("process %d:pid:%u, IPU Util:%u %%, JPU Util:%u %%, VPU Decoder Util:%u %%, VPU Encoder Util:%u %%, Memory Util:%u %%\n",
              i, (procUtil+i)->pid, (procUtil + i)->ipuUtil, (procUtil + i)->jpuUtil,
              (procUtil + i)->vpuDecUtil,
              (procUtil + i)->vpuEncUtil,
              (procUtil + i)->memUtil);
      }
    }
    free(procUtil);

    if (nameInfo.id != MLU220_EDGE) {
      cndevAffinity_t a;
      a.version = CNDEV_VERSION_5;
      ret = cndevGetDeviceAffinity(&a, devHandle);
      if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
        printf("cndevGetDeviceAffinity is not supported.\n");
      } else {
        cndevCheckErrors(ret);
        printf("cpuCount:%d\n", a.cpuCount);
        int count = a.cpuCount / 32;
        if (a.cpuCount % 32)
          count++;
        for(int i = 0; i < count; ++i) {
          printf("device affinity bit map %d: %x\n", i, a.cpuAffinityBitMap[i]);
        }
        cndevCheckErrors(cndevClearCurrentThreadAffinity(CNDEV_VERSION_5, devHandle));
        cndevCheckErrors(cndevSetCurrentThreadAffinity(CNDEV_VERSION_5, devHandle));
        //  sleep(3);
      }

      cndevTopologyRelationship_t rel;
      rel.version = CNDEV_VERSION_5;
      ret = cndevTopologyGetRelationship(&rel, card, devHandle);
      if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
        printf("cndevTopologyGetRelationship is not supported.\n");
      } else {
        cndevCheckErrors(ret);
        printf("relation is: %d\n", rel.relation);
      }

      __uint64_t arrayCount = 10;
      __uint64_t *devIdArray = NULL;
      devIdArray = (__uint64_t*) malloc(arrayCount * sizeof(__uint64_t));

      ret = cndevTopologyGetNearestDevices(CNDEV_VERSION_5, SELF, &arrayCount, devIdArray, devHandle);
      if (ret == CNDEV_ERROR_INSUFFICIENT_SPACE) {
        devIdArray = (__uint64_t*) realloc(devIdArray, arrayCount * sizeof(__uint64_t));
        ret = cndevTopologyGetNearestDevices(CNDEV_VERSION_5, SELF, &arrayCount, devIdArray, devHandle);
      }

      if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
        printf("cndevTopologyGetNearestDevices is not supported.\n");
      } else {
        cndevCheckErrors(ret);
        for (__uint64_t i = 0; i < arrayCount; ++i) {
          printf("relationship between %d and %ld is self\n", card, *(devIdArray + i));
        }
        free(devIdArray);
      }
      cndevCurrentPCIInfo_t currentPCI;
      currentPCI.version = CNDEV_VERSION_5;
      ret = cndevGetCurrentPCIInfo(&currentPCI, devHandle);
      if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
        printf("cndevGetCurrentPCIInfo is not supported.\n");
      } else {
        cndevCheckErrors(ret);
        printf("current width is x%d\n", currentPCI.currentWidth);
        switch (currentPCI.currentSpeed) {
        case 1:
          printf("current link speed is 2.5 GT/s\n"); break;
        case 2:
          printf("current link speed is 5 GT/s\n"); break;
        case 3:
          printf("current link speed is 8 GT/s\n"); break;
        case 4:
          printf("current link speed is 16 GT/s\n"); break;
        default:
          printf("current link speed is UNKNOWN\n"); break;
        }
      }
    }

    cndevNUMANodeId_t numanodeid;
    numanodeid.version = CNDEV_VERSION_5;
    ret = cndevGetNUMANodeIdByDevId(&numanodeid, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetNUMANodeIdByDevId is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("card %d NUMA Node ID: %d\n", card, numanodeid.nodeId);
    }

    int port_number = cndevGetMLULinkPortNumber(devHandle);
    printf("card %d port number: %d\n", card, port_number);
    if (port_number) {
      cndevMLULinkDevSN_t devinfo;
      devinfo.version = CNDEV_VERSION_5;
      ret = cndevGetMLULinkDevSN(&devinfo, devHandle);
      if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
        printf("cndevGetMLULinkDevSN is not supported\n");
      } else {
        cndevCheckErrors(ret);
        for (int i = 0; i < port_number; ++i) {
          printf("card %d MLU-Link %d device MC SN %012lx\n", card, i, devinfo.mlulinkMcSn[i]);
          printf("card %d MLU-Link %d device BA SN %012lx\n", card, i, devinfo.mlulinkBaSn[i]);
        }
      }

      // MLU-Link
      for (int link = 0; link < port_number; ++link) {
        cndevMLULinkStatus_t mlulink_status;
        mlulink_status.version = CNDEV_VERSION_5;
        ret = cndevGetMLULinkStatus(&mlulink_status, card, link);
        if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
          printf("cndevGetMLULinkStatus is not supported\n");
          break;
        }
        printf("card %d link %d MLU-Link status: %d\n", card, link, mlulink_status.isActive);
        printf("card %d link %d Serdes status: %d\n", card, link, mlulink_status.serdesState);
        if (!mlulink_status.isActive)
          continue;

        cndevMLULinkVersion_t mlulink_version;
        mlulink_version.version = CNDEV_VERSION_5;
        cndevCheckErrors(cndevGetMLULinkVersion(&mlulink_version, card, link));
        printf("card %d link %d MLU-Link version: %d.%d.%d\n", card, link, mlulink_version.majorVersion,
               mlulink_version.minorVersion, mlulink_version.buildVersion);

        cndevMLULinkCapability_t mlulink_capability;
        mlulink_capability.version = CNDEV_VERSION_5;
        cndevCheckErrors(cndevGetMLULinkCapability(&mlulink_capability, card, link));
        printf("card %d link %d MLU-Link p2pTransfer: %d\n", card, link, mlulink_capability.p2pTransfer);
        printf("card %d link %d MLU-Link interlakenSerdes: %d\n", card, link, mlulink_capability.interlakenSerdes);

        cndevMLULinkSpeed_t mlulink_speed;
        mlulink_speed.version = CNDEV_VERSION_5;
        cndevCheckErrors(cndevGetMLULinkSpeedInfo(&mlulink_speed, card, link));
        if (mlulink_speed.speedFormat == SPEED_FMT_NRZ)
          printf("card %d link %d MLU-Link FMT_NRZ speed: %.03f GB/s\n", card, link, mlulink_speed.speedValue);
        if (mlulink_speed.speedFormat == SPEED_FMT_PM4)
          printf("card %d link %d MLU-Link FMT_PM4 speed: %.03f GB/s\n", card, link, mlulink_speed.speedValue);

        cndevMLULinkCounter_t mlulink_counter;
        mlulink_counter.version = CNDEV_VERSION_5;
        ret = cndevGetMLULinkCounter(&mlulink_counter, card, link);
        if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
          printf("cndevGetMLULinkCounter is not supported\n");
        } else {
          printf("card %d link %d CNTR_RD_BYTE Counter:%ld\n", card, link, mlulink_counter.cntrReadByte);
          printf("card %d link %d CNTR_RD_PKG Counter:%ld\n", card, link, mlulink_counter.cntrReadPackage);
          printf("card %d link %d CNTR_WR_BYTE Counter:%ld\n", card, link, mlulink_counter.cntrWriteByte);
          printf("card %d link %d CNTR_WR_PKG Counter:%ld\n", card, link, mlulink_counter.cntrWritePackage);
          printf("card %d link %d ERR_RPY Counter:%ld\n", card, link, mlulink_counter.errReplay);
          printf("card %d link %d ERR_FTL Counter:%ld\n", card, link, mlulink_counter.errFatal);
          printf("card %d link %d ERR_ECC_DEB Counter:%ld\n", card, link, mlulink_counter.errEccDouble);
          printf("card %d link %d ERR_CRC24 Counter:%ld\n", card, link, mlulink_counter.errCRC24);
          printf("card %d link %d ERR_CRC32 Counter:%ld\n", card, link, mlulink_counter.errCRC32);
          printf("card %d link %d ERR_CORR Counter:%ld\n", card, link, mlulink_counter.errCorrected);
          printf("card %d link %d ERR_UNCORR Counter:%ld\n", card, link, mlulink_counter.errUncorrected);
          printf("card %d link %d ERR_READ_PDG Counter:%ld\n", card, link, mlulink_counter.errReadPackage);
          printf("card %d link %d ERR_WRITE_PKG Counter:%ld\n", card, link, mlulink_counter.errWritePackage);
          printf("card %d link %d ERR_ILLEGAL_ACCESS Counter:%ld\n", card, link, mlulink_counter.errIllegalAccess);
        }

        cndevMLULinkRemoteInfo_t mlulink_remote;
        mlulink_remote.version = CNDEV_VERSION_5;
        ret = cndevGetMLULinkRemoteInfo(&mlulink_remote, card, link);
        if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
          printf("cndevGetMLULinkRemoteInfo is not supported\n");
        } else {
          cndevCheckErrors(ret);
          printf("card %d link %d is ip valid:%d\n", card, link, mlulink_remote.isIpValid);
          printf("card %d link %d connect type:%d\n", card, link, mlulink_remote.connectType);
          printf("card %d link %d MC SN:%012lx\n", card, link, mlulink_remote.mcSn);
          printf("card %d link %d BA SN:%012lx\n", card, link, mlulink_remote.baSn);
          printf("card %d link %d slot ID:%d\n", card, link, mlulink_remote.slotId);
          printf("card %d link %d port id:%d\n", card, link, mlulink_remote.portId);
          printf("card %d link %d device ip version:%d\n", card, link, mlulink_remote.devIpVersion);
          printf("card %d link %d ncsUUID:%lu\n", card, link, mlulink_remote.ncsUUID64);
          printf("card %d link %d UUID:%s\n", card, link, mlulink_remote.uuid);
        }

        cndevMLULinkPortMode_t mode;
        mode.version = CNDEV_VERSION_5;
        ret = cndevGetMLULinkPortMode(&mode, card, link);
        if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
          printf("cndevGetMLULinkPortMode is not supported.\n");
        } else {
          cndevCheckErrors(ret);
          if (mode.mode == PORT_WORK_MODE_MLULINK) {
            printf("card %d link %d mode is MLULINK\n", card, link);
          } else if (mode.mode == PORT_WORK_MODE_MLULINK_OVER_ROCE) {
            printf("card %d link %d mode is MLULinkOverRoCE\n", card, link);
            for (int j = 1; j < CNDEV_MLULINK_FIELD_NUM; j++) {
              cndevMLULinkOverRoCECtrl_t ctrl;
              ctrl.version = CNDEV_VERSION_5;
              ctrl.field = (cndevMLULinkOverRoCEFieldEnum_t)j;
              cndevCheckErrors(cndevGetMLULinkOverRoCECtrl(&ctrl, card, link));
              printf("card %d link %d field:%d 's value is %d\n", card, link, j, ctrl.value);
            }
          }
        }

        cndevMLULinkPortIP_t portip;
        portip.version = CNDEV_VERSION_5;
        ret = cndevGetMLULinkPortIP(&portip, card, link);
        if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
          printf("cndevGetMLULinkPortIP is not supported.\n");
        } else {
          cndevCheckErrors(ret);
          if (portip.ipVersion == 4 || portip.ipVersion == 6)
            printf("card %d link %d ip:%s\n", card, link, portip.ip);
        }
      }
    }
    cndevUUID_t uuidInfo;
    uuidInfo.version = CNDEV_VERSION_5;
    ret = cndevGetUUID(&uuidInfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetUUID is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("UUID: %s\n", uuidInfo.uuid);
      printf("NCS UUID: %lu\n", uuidInfo.ncsUUID64);
    }

    cndevRetiredPageInfo_t retiredpageinfo;
    retiredpageinfo.version = CNDEV_VERSION_5;
    retiredpageinfo.cause = CNDEV_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR;
    // retiredpageinfo.cause = CNDEV_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS;
    ret = cndevGetRetiredPages(&retiredpageinfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetRetiredPages is not supported\n");
    } else {
      cndevCheckErrors(ret);
      for (size_t i = 0; i < retiredpageinfo.pageCount; ++i) {
        printf("retiredPages Address %ld: 0x%012lx\n", i, retiredpageinfo.pageAddress[i]);
      }
    }
    cndevRetiredPageStatus_t retiredpagestatus;
    retiredpagestatus.version = CNDEV_VERSION_5;
    ret = cndevGetRetiredPagesStatus(&retiredpagestatus, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetRetiredPagesStatus is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("retiredPages is pending: %d\n", retiredpagestatus.isPending);
      printf("retiredPages is failed: %d\n", retiredpagestatus.isFailure);
    }
    cndevRemappedRow_t rows;
    rows.version = CNDEV_VERSION_5;
    ret = cndevGetRemappedRows(&rows, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetRemappedRows is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("corrected rows: %d\n", rows.correctRows);
      printf("uncorrected rows: %d\n", rows.uncorrectRows);
      printf("failed rows: %d\n", rows.failedRows);
      printf("pending rows: %d\n", rows.pendingRows);
    }
    cndevRetiredPageOperation_t operation;
    operation.version = CNDEV_VERSION_5;
    ret = cndevGetRetiredPagesOperation(&operation, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetRetiredPagesOperation is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("retirementpages operation: %d\n", operation.retirePageOption);
    }

    cndevChipId_t chipid;
    chipid.version = CNDEV_VERSION_5;
    ret = cndevGetChipId(&chipid, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetChipId is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("card %d chip id :%d\n", card, chipid.chipId);
    }

    cndevCRCInfo_t crcinfo;
    crcinfo.version = CNDEV_VERSION_5;
    ret = cndevGetCRCInfo(&crcinfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetCRCInfo is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("CRC: d2d crc error:%lu, d2d crc error overflow:%lu\n", crcinfo.die2dieCRCError, crcinfo.die2dieCRCErrorOverflow);
    }

    cndevDDRInfo_t ddrinfo;
    ddrinfo.version = CNDEV_VERSION_5;
    ret = cndevGetDDRInfo(&ddrinfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetDDRInfo is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("DDR DataWidth: %d Bit\n", ddrinfo.dataWidth);
      printf("DDR BandWidth: %d.%d B/ns\n", ddrinfo.bandWidth, ddrinfo.bandWidthDecimal);
    }

    // get card[x]'s supported ipu frequency
    // in most cases, the number of supported ipu frequency running on a card will not exceed 10
    // but if cndevSupportedIpuFrequency_t's space is not enough, CNDEV_ERROR_INSUFFICIENT_SPACE will be returned
    unsigned int ipucount = 10;
    cndevSupportedIpuFrequency_t *supportedipufreq = NULL;
    supportedipufreq = (cndevSupportedIpuFrequency_t *) malloc(ipucount * sizeof(cndevSupportedIpuFrequency_t));
    supportedipufreq->version = CNDEV_VERSION_5;
    ret = cndevGetSupportedIpuFrequency(&ipucount, supportedipufreq, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetSupportedIpuFrequency is not supported\n");
    } else {
      // if ret is CNDEV_ERROR_INSUFFICIENT_SPACE, should get again
      while (ret == CNDEV_ERROR_INSUFFICIENT_SPACE) {
        supportedipufreq = (cndevSupportedIpuFrequency_t *) realloc(supportedipufreq, ipucount * sizeof(cndevSupportedIpuFrequency_t));
        ret = cndevGetSupportedIpuFrequency(&ipucount, supportedipufreq, devHandle);
      }
      cndevCheckErrors(ret);
      printf("supported ipu frequency count: %d\n", ipucount);
      for (unsigned int i = 0; i < ipucount; i++) {
        printf("supported ipu frequency %d : %d MHz\n", i, (supportedipufreq + i)->supportedIpuFreq);
      }
    }
    free(supportedipufreq);

    // set Ipu Frequency
    // cndevSetIpuFrequency_t setipufreq;
    // setipufreq.version = CNDEV_VERSION_5;
    // setipufreq.minIpuFreq = 600;
    // setipufreq.maxIpuFreq = 600;
    // cndevCheckErrors(cndevSetIpuFrequency(&setipufreq, devHandle));

    // get xid error
    cndevXidError_t xiderr;
    xiderr.version = CNDEV_VERSION_5;
    ret = cndevGetXidError(&xiderr, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetXidError is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Last XID:%d, %s\n",xiderr.lastXid, cndevGetXidErrorString(xiderr.lastXid));
      printf("total Xid count:%d\n", xiderr.totalXidCount);
      for (size_t i = 0; i < xiderr.totalXidCount; ++i) {
        printf("total xid:%u\n", xiderr.totalXid[i]);
      }
    }
    
    // get xid error switch status
    cndevXidStatus_t xidstatus;
    xidstatus.version = CNDEV_VERSION_5;
    ret = cndevGetXidStatus(&xidstatus, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetXidStatus is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      for (int i = 0; i < CNDEV_XID_MAX_COUNT; ++i) {
        printf("xid %d switch status:%s\n", i,
        (xidstatus.xidStatus[i] == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
      }
    }

    // clear xid error
    cndevSetXid_t clearXid;
    clearXid.version = CNDEV_VERSION_5;
    // clearXid.isSelectAll = 1;  // clear all xid error
    clearXid.selectedXid = CNDEV_XID_ECC_ERROR;
    ret = cndevClearXidError(&clearXid, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevClearXidError is not supported.\n");
    } else if (ret == CNDEV_ERROR_NO_PERMISSION) {
      printf("cndevClearXidError permission denied.\n");
    } else {
      cndevCheckErrors(ret);
      printf("cndevClearXidError Success.\n");
    }

    // set xid error
    cndevSetXid_t setxid;
    setxid.version = CNDEV_VERSION_5;
    // setxid.isSelectAll = 1;  // set all xid error
    setxid.selectedXid = CNDEV_XID_ECC_ERROR;
    setxid.setXidStatus = CNDEV_FEATURE_ENABLED;
    ret = cndevSetXidStatus(&setxid, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevSetXidStatus is not supported.\n");
    } else if (ret == CNDEV_ERROR_NO_PERMISSION) {
      printf("cndevSetXidStatus permission denied.\n");
    } else {
      cndevCheckErrors(ret);
      printf("cndevSetXidStatus Success.\n");
    }

    // get overtemperature information
    cndevOverTemperatureInfo_t overtempInfo;
    overtempInfo.version = CNDEV_VERSION_5;
    ret = cndevGetOverTemperatureInfo(&overtempInfo, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetOverTemperatureInfo is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("OverTemperature power-off Counter:%d\n", overtempInfo.powerOffCounter);
      printf("OverTemperature underclocking Counter:%d\n", overtempInfo.underClockCounter);
      printf("OverTemperature power-off temp:%d\n", overtempInfo.powerOffTemp);
      printf("OverTemperature underclocking temp:%d\n", overtempInfo.underClockTemp);
    }
    
    // get device computing power
    cndevDeviceMaxPerformance_t maxperformance;
    maxperformance.version = CNDEV_VERSION_5;
    ret = cndevGetDeviceMaxPerformance(&maxperformance, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetDeviceMaxPerformance is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Peak INT4 Tensor MACs per cycle per core:%lu\n", maxperformance.int4Tensor);
      printf("Peak INT8 Tensor MACs per cycle per core:%lu\n", maxperformance.int8Tensor);
      printf("Peak INT16 Tensor MACs per cycle per core:%lu\n", maxperformance.int16Tensor);
      printf("Peak FP16 Tensor MACs per cycle per core:%lu\n", maxperformance.fp16Tensor);
      printf("Peak BF16 Tensor MACs per cycle per core:%lu\n", maxperformance.bf16Tensor);
      printf("Peak FP32 Tensor MACs per cycle per core:%lu\n", maxperformance.fp32Tensor);

      printf("Peak FP16 Vector OP per cycle per core:%lu\n", maxperformance.fp16Vector);
      printf("Peak BF16 Vector OP per cycle per core:%lu\n", maxperformance.bf16Vector);
      printf("Peak FP32 Vector OP per cycle per core:%lu\n", maxperformance.fp32Vector);
      printf("Peak INT8 Vector OP per cycle per core:%lu\n", maxperformance.int8Vector);
      printf("Peak INT16 Vector OP per cycle per core:%lu\n", maxperformance.int16Vector);
    }

    // get performance throttle reason
    cndevPerformanceThrottleReason_t reason;
    reason.version = CNDEV_VERSION_5;
    ret = cndevGetPerformanceThrottleReason(&reason, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetPerformanceThrottleReason is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Performance throttle reason:\n");
      printf("tdp:%s\n", (reason.tdp == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
      printf("powerSetting:%s\n", (reason.powerSetting == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
      printf("clockSetting:%s\n", (reason.clockSetting == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
      printf("frequencyLocked:%s\n", (reason.frequencyLocked == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
      printf("powerBrake:%s\n", (reason.powerBrake == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
      printf("thermalSlowdown:%s\n", (reason.thermalSlowdown == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
    }

    // get compute mode
    cndevComputeMode_t cpmode;
    cpmode.version = CNDEV_VERSION_5;
    ret = cndevGetComputeMode(&cpmode, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetComputeMode is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Compute mode: %d\n", cpmode.mode);
    }

    // set compute mode
    cpmode.mode = CNDEV_COMPUTEMODE_DEFAULT;  // CNDEV_COMPUTEMODE_EXCLUSIVE;
    ret = cndevSetComputeMode(&cpmode, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevSetComputeMode is not supported.\n");
    } else if (ret == CNDEV_ERROR_NO_PERMISSION) {
      printf("cndevSetComputeMode permission denied.\n");
    } else {
      cndevCheckErrors(ret);
      printf("cndevSetComputeMode Success.\n");
    }

    // get PCIe replay counter
    cndevPcieReplayCounter_t replay_counter;
    replay_counter.version = CNDEV_VERSION_5;
    ret = cndevGetPcieReplayCounter(&replay_counter, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetPcieReplayCounter is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("PCIe replay counter: %d\n", replay_counter.counter);
    }

    // get parity error
    cndevParityError_t parity_error;
    parity_error.version = CNDEV_VERSION_5;
    ret = cndevGetParityError(&parity_error, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetParityError is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Parity Error counter: %d\n", parity_error.counter);
    }

    // get docker param
    cndevDockerParam_t docker_param;
    docker_param.version = CNDEV_VERSION_5;
    ret = cndevGetDockerParam(&docker_param, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetDockerParam is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Docker Param: %s\n", docker_param.dockerParam);
    }

    // get mim mode
    cndevMimMode_t mim_mode;
    mim_mode.version = CNDEV_VERSION_5;
    ret = cndevGetMimMode(&mim_mode, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetMimMode is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("Mim Mode: %s\n", (mim_mode.mimMode == CNDEV_FEATURE_ENABLED) ? "ON" : "OFF");
    }

    // set mim mode
    mim_mode.mimMode = CNDEV_FEATURE_DISABLED;
    ret = cndevSetMimMode(&mim_mode, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevSetMimMode is not supported.\n");
    } else if (ret == CNDEV_ERROR_NO_PERMISSION) {
      printf("cndevSetMimMode permission denied.\n");
    } else {
      cndevCheckErrors(ret);
      printf("cndevSetMimMode Success.\n");
    }

    // get vf information
    cndevCardVfState_t vfstate;
    vfstate.version = CNDEV_VERSION_5;
    ret = cndevGetCardVfState(&vfstate, devHandle);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetCardVfState is not supported\n");
    } else {
      cndevCheckErrors(ret);
      printf("card %d vf state : %d\n", card, vfstate.vfState);
    }
    int num = 0;
    while (vfstate.vfState) {
      num++;
      if (vfstate.vfState & 0x1) {
        cndevMemoryInfo_t vfmemInfo;
        vfmemInfo.version = CNDEV_VERSION_5;
        cndevCheckErrors(cndevGetMemoryUsage(&vfmemInfo, num << 8 | devHandle));
        printf("Card %d vf %d Phy mem total:%ldMiB, Phy mem used:%ldMiB, Vir mem total:%ldMiB, Vir mem used:%ldMiB\n",
            card, num, vfmemInfo.physicalMemoryTotal, vfmemInfo.physicalMemoryUsed,
            vfmemInfo.virtualMemoryTotal, vfmemInfo.virtualMemoryUsed);
        for (int i = 0; i < vfmemInfo.channelNumber; ++i) {
          printf("Channel %d mem used:%ldMib\n", i, vfmemInfo.channelMemoryUsed[i]);
        }
      }
      vfstate.vfState >>= 1;
    }
  }  // for card_num

#ifndef WIN32
  // Topology information
  for (int j = 0; j < get_nprocs_conf(); ++j) {
    __uint64_t *arrayCount = NULL;
    arrayCount = (__uint64_t *)malloc(sizeof(__uint64_t));
    *arrayCount = 10;
    __uint64_t *array = NULL;
    array = (__uint64_t *)malloc((*arrayCount) * sizeof(__uint64_t));
    ret = cndevTopologyGetCpuRelatedDevices(CNDEV_VERSION_5, j, arrayCount, array);
    if (ret == CNDEV_ERROR_INSUFFICIENT_SPACE) {
      array = (__uint64_t *)realloc(array, (*arrayCount) * sizeof(__uint64_t));
      ret = cndevTopologyGetCpuRelatedDevices(CNDEV_VERSION_5, j, arrayCount, array);
    }
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevTopologyGetCpuRelatedDevices is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      for (__uint64_t i = 0; i < *arrayCount; ++i) {
        printf("devices %ld is related to cpu %d\n", *(array + i), j);
      }
    }
    free(array);
    free(arrayCount);
  }

  cndevTopologyNode_t *virtualRoot;
  cndevCheckErrors(cndevTopologyGetVirtualRootNode(CNDEV_VERSION_5, &virtualRoot));
  if (virtualRoot != NULL) printf("virtual root class name is %s\n", virtualRoot->className);

  cndevTopologyNode_t *treeNode;
  // for example
  int domain = 0;
  int bus = 0;
  int device = 0;
  int function = 0;
  cndevRet_t r = cndevGetNodeByBDF(CNDEV_VERSION_5, &treeNode, domain, bus, device, function);
  if (r == CNDEV_ERROR_NOT_SUPPORTED) {
    printf("cndevGetNodeByBDF is not supported.\n");
  } else if (r == CNDEV_ERROR_INVALID_ARGUMENT) {
    printf("cndevGetNodeByBDF failed.Maybe the node does not exist.\n");
  } else {
    cndevCheckErrors(r);
    printf("the class name of device %04x:%02x:%02x.%x is %s\n", domain, bus, device, function, treeNode->className);
    printf("the subsystem vendor of device %04x:%02x:%02x.%x is 0x%x\n", domain, bus, device, function,
           treeNode->subsystemVendor);
    printf("the vendor of device %04x:%02x:%02x.%x is 0x%x\n", domain, bus, device, function, treeNode->vendor);
    printf("the device id of device %04x:%02x:%02x.%x is 0x%04x\n", domain, bus, device, function, treeNode->deviceId);
    printf("the subsystem id of device %04x:%02x:%02x.%x is 0x%04x\n", domain, bus, device, function, treeNode->subsystemId);
  }
  cndevNUMANodeId_t numaNode;
  numaNode.version = CNDEV_VERSION_5;
  r = cndevGetNUMANodeIdByTopologyNode(&numaNode, treeNode);
  if (r == CNDEV_ERROR_NOT_SUPPORTED) {
    printf("cndevGetNUMANodeIdByTopologyNode is not supported.\n");
  } else if (r == CNDEV_ERROR_INVALID_ARGUMENT) {
    printf("cndevGetNUMANodeIdByTopologyNode failed.Maybe the node does not exist.\n");
  } else {
    cndevCheckErrors(r);
    printf("the NUMA node id of device %04x:%02x:%02x.%x is %d\n", domain, bus, device, function, numaNode.nodeId);
  }

  // cndevCapabilityInfo_t capInfo;
  // capInfo.version = CNDEV_VERSION_5;
  // capInfo.id = PCI_CAP_ID_VNDR;
  // cndevCheckErrors(cndevGetNodeCapabilityInfo(&capInfo, treeNode));
  // printf("the cap of 0000:00:00.0 is 0X%04x\n", capInfo.cap);
  cndevPCIeInfo_t deviceId;
  deviceId.version = CNDEV_VERSION_5;
  ret = cndevGetPCIeInfo(&deviceId, 0);
  if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
    printf("cndevGetPCIeInfo is not supported.\n");
  } else {
    cndevCheckErrors(ret);
    int devId;
    cndevCheckErrors(
        cndevGetDevIdByBDF(CNDEV_VERSION_5, &devId, deviceId.domain, deviceId.bus, deviceId.device, deviceId.function));
    printf("the dev id of device %04x:%02x:%02x.%x is %d\n", deviceId.domain, deviceId.bus, deviceId.device, deviceId.function,
           devId);
    cndevTopologyNode_t *a;
    int card = 0;
    ret = cndevGetNodeByDevId(CNDEV_VERSION_5, &a, 0);
    if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
      printf("cndevGetNodeByDevId is not supported.\n");
    } else {
      cndevCheckErrors(ret);
      printf("the class name of card %d is %s\n", card, a->className);
    }
  }

  int id = 8;
  ret = cndevTopologyTraverseTree(CNDEV_VERSION_5, printName, &id);
  if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
    printf("cndevTopologyTraverseTree is not supported.\n");
  } else {
    cndevCheckErrors(ret);
  }

  int arrayCount = 10;
  cndevTopologyNode_t *Array = NULL;
  Array = (cndevTopologyNode_t *)malloc((arrayCount) * sizeof(cndevTopologyNode_t));
  ret = cndevGetNodeByDeviceName(CNDEV_VERSION_5, &arrayCount, Array, "Intel Corporation");
  if (ret == CNDEV_ERROR_NOT_SUPPORTED) {
    printf("cndevGetNodeByDeviceName is not supported.\n");
  } else if (ret == CNDEV_ERROR_INSUFFICIENT_SPACE) {
    Array = (cndevTopologyNode_t *)realloc(Array, (arrayCount) * sizeof(cndevTopologyNode_t));
    ret = cndevGetNodeByDeviceName(CNDEV_VERSION_5, &arrayCount, Array, "Intel Corporation");
  } else {
    cndevCheckErrors(ret);
    for (int i = 0; i < arrayCount; ++i) {
      printf("node :%04x:%02x:%02x.%x\n", (Array + i)->domain, (Array + i)->bus, (Array + i)->device, (Array + i)->function);
    }
    free(Array);
  }
#endif
  cndevRet_t last_error;
  last_error = cndevGetLastError();
  printf("lastError value is %d\n", last_error);
  // release libcndev
  cndevRelease();
}

